{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "8klpyPsOv3hS"
   },
   "outputs": [],
   "source": [
    "!git clone https://github.com/huggingface/transformers\n",
    "%cd transformers\n",
    "!pip install .\n",
    "!pip install -r ./examples/multiple-choice/requirements.txt\n",
    "%cd ..\n",
    "!mv ./transformers/examples/multiple-choice/run_swag.py ./transformers/examples/multiple-choice/run_swag.py.bak\n",
    "!cp run_swag.py ./transformers/examples/multiple-choice/run_swag.py\n",
    "\n",
    "!mkdir -pv swag\n",
    "!wget https://raw.githubusercontent.com/rowanz/swagaf/master/data/train.csv -O swag/train.csv\n",
    "# !wget https://raw.githubusercontent.com/rowanz/swagaf/master/data/val.csv -O swag/val.csv\n",
    "# !wget https://raw.githubusercontent.com/rowanz/swagaf/master/data/test.csv -O swag/test.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DEBUG = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "wwpiCMuSfC7u"
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "import pandas as pd\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6315, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Content</th>\n",
       "      <th>Questions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>奉和袭美抱疾杜门见寄次韵  陆龟蒙虽失春城醉上期，下帷裁遍未裁诗。因吟郢岸百亩蕙，欲采商崖三...</td>\n",
       "      <td>[{'Question': '下列对这首诗的理解和赏析，不正确的一项是', 'Choices...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>隆冬之际，西伯利亚的寒流（笼罩/席卷）欧亚大陆，狂风肆虐，草木凋凌，而那些春天的元素——温暖...</td>\n",
       "      <td>[{'Question': '1. 文中“肆虐”“凋凌”“昼夜兼程”“版图”“稚嫩”“嫣然”...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>材料一：日前，中国科学院在京召开新闻发布会对外宣布“墨子号”进行科学实验卫星提前并圆满实现全...</td>\n",
       "      <td>[{'Question': '下列对材料相关内容的理解，不正确的一项是', 'Choices...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>纯礼字彝叟以父仲淹荫知陵台令兼永安县永昭陵建京西转运使配木石砖甓及工徒于一路独永安不受令使者...</td>\n",
       "      <td>[{'Question': '下列对文中“纯礼字彝叟以父仲淹荫知陵台令兼永安县永昭陵建京西转...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>中国传统音乐包括民间音乐、宗教音乐、文人音乐、宫廷音乐等类别，其中文人音乐的代表主要就是古琴...</td>\n",
       "      <td>[{'Question': '下面填入文中括号内的语句，衔接最恰当的一项是', 'Choic...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID                                            Content  \\\n",
       "0   1  奉和袭美抱疾杜门见寄次韵  陆龟蒙虽失春城醉上期，下帷裁遍未裁诗。因吟郢岸百亩蕙，欲采商崖三...   \n",
       "1   2  隆冬之际，西伯利亚的寒流（笼罩/席卷）欧亚大陆，狂风肆虐，草木凋凌，而那些春天的元素——温暖...   \n",
       "2   3  材料一：日前，中国科学院在京召开新闻发布会对外宣布“墨子号”进行科学实验卫星提前并圆满实现全...   \n",
       "3   4  纯礼字彝叟以父仲淹荫知陵台令兼永安县永昭陵建京西转运使配木石砖甓及工徒于一路独永安不受令使者...   \n",
       "4   5  中国传统音乐包括民间音乐、宗教音乐、文人音乐、宫廷音乐等类别，其中文人音乐的代表主要就是古琴...   \n",
       "\n",
       "                                           Questions  \n",
       "0  [{'Question': '下列对这首诗的理解和赏析，不正确的一项是', 'Choices...  \n",
       "1  [{'Question': '1. 文中“肆虐”“凋凌”“昼夜兼程”“版图”“稚嫩”“嫣然”...  \n",
       "2  [{'Question': '下列对材料相关内容的理解，不正确的一项是', 'Choices...  \n",
       "3  [{'Question': '下列对文中“纯礼字彝叟以父仲淹荫知陵台令兼永安县永昭陵建京西转...  \n",
       "4  [{'Question': '下面填入文中括号内的语句，衔接最恰当的一项是', 'Choic...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_json(\"raw_data/train.json\")\n",
    "\n",
    "print(train.shape)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "隆冬之际，西伯利亚的寒流（笼罩/席卷）欧亚大陆，狂风肆虐，草木凋凌，而那些春天的元素——温暖、雨水、绿叶、鲜花，都集结在位于热带的海南岛。海南岛就像是一艘花船，（系/停）在雷州半岛上，满载寒冬大陆的梦幻和想象。每年，从广州向漠河，春天昼夜兼程，都要进行一次生命版图的（扩展/扩充）。他像赤足奔跑的孩子，一路上用稚嫩的声音轻轻呼唤，于是万物苏醒，盛装应和，可谓“东风好作阳和使，      。”迢迢旅途中，气候的巨大差异，导致众多物种中只能有限地参与这一盛会。木棉花花朵硕大，是南国花中豪杰，“一声铜鼓催开，千树珊瑚齐列，”但她终究无法走出岭南。当春天行经长江、黄河流域时，出场的是桃花、杏花等新主角，“桃花嫣然出篱笑，          ”，然而她们却无法追随春天深入雪国，陆续抱憾退出，随后登场的便是白杨、连翘等北国耐寒植物。\n",
      "[{'Question': '1. 文中“肆虐”“凋凌”“昼夜兼程”“版图”“稚嫩”“嫣然”“抱憾退出”的词语中，有错别字的一项是', 'Choices': ['A. 肆虐  凋凌', 'B. 集结  昼夜兼程', 'C. 版图  稚嫩', 'D. 嫣然  抱憾退出'], 'Answer': 'A', 'Q_id': '000201'}, {'Question': '依次选用文中括号里的词语，最恰当的一项是', 'Choices': ['A. 席卷  系  扩展', 'B. 笼罩  停  扩展', 'C. 席卷  停  扩充', 'D. 笼罩  系  扩充'], 'Answer': 'A', 'Q_id': '000202'}]\n"
     ]
    }
   ],
   "source": [
    "print(train['Content'].iloc[1])\n",
    "print(train['Questions'].iloc[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>video-id</th>\n",
       "      <th>fold-ind</th>\n",
       "      <th>startphrase</th>\n",
       "      <th>sent1</th>\n",
       "      <th>sent2</th>\n",
       "      <th>gold-source</th>\n",
       "      <th>ending0</th>\n",
       "      <th>ending1</th>\n",
       "      <th>ending2</th>\n",
       "      <th>ending3</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>anetv_jkn6uvmqwh4</td>\n",
       "      <td>3416</td>\n",
       "      <td>Members of the procession walk down the street...</td>\n",
       "      <td>Members of the procession walk down the street...</td>\n",
       "      <td>A drum line</td>\n",
       "      <td>gold</td>\n",
       "      <td>passes by walking down the street playing thei...</td>\n",
       "      <td>has heard approaching them.</td>\n",
       "      <td>arrives and they're outside dancing and asleep.</td>\n",
       "      <td>turns the lead singer watches the performance.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>anetv_jkn6uvmqwh4</td>\n",
       "      <td>3417</td>\n",
       "      <td>A drum line passes by walking down the street ...</td>\n",
       "      <td>A drum line passes by walking down the street ...</td>\n",
       "      <td>Members of the procession</td>\n",
       "      <td>gen</td>\n",
       "      <td>are playing ping pong and celebrating one left...</td>\n",
       "      <td>wait slowly towards the cadets.</td>\n",
       "      <td>continues to play as well along the crowd alon...</td>\n",
       "      <td>continue to play marching, interspersed.</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>anetv_jkn6uvmqwh4</td>\n",
       "      <td>3415</td>\n",
       "      <td>A group of members in green uniforms walks wav...</td>\n",
       "      <td>A group of members in green uniforms walks wav...</td>\n",
       "      <td>Members of the procession</td>\n",
       "      <td>gold</td>\n",
       "      <td>pay the other coaches to cheer as people this ...</td>\n",
       "      <td>walk down the street holding small horn brass ...</td>\n",
       "      <td>is seen in the background.</td>\n",
       "      <td>are talking a couple of people playing a game ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>anetv_jkn6uvmqwh4</td>\n",
       "      <td>3417</td>\n",
       "      <td>A drum line passes by walking down the street ...</td>\n",
       "      <td>A drum line passes by walking down the street ...</td>\n",
       "      <td>Members of the procession</td>\n",
       "      <td>gen</td>\n",
       "      <td>are playing ping pong and celebrating one left...</td>\n",
       "      <td>wait slowly towards the cadets.</td>\n",
       "      <td>makes a square call and ends by jumping down i...</td>\n",
       "      <td>play and go back and forth hitting the drums w...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>anetv_Bri_myFFu4A</td>\n",
       "      <td>2408</td>\n",
       "      <td>The person plays a song on the violin. The man</td>\n",
       "      <td>The person plays a song on the violin.</td>\n",
       "      <td>The man</td>\n",
       "      <td>gold</td>\n",
       "      <td>finishes the song and lowers the instrument.</td>\n",
       "      <td>hits the saxophone and demonstrates how to pro...</td>\n",
       "      <td>finishes massage the instrument again and cont...</td>\n",
       "      <td>continues dancing while the man gore the music...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0           video-id  fold-ind  \\\n",
       "0           0  anetv_jkn6uvmqwh4      3416   \n",
       "1           1  anetv_jkn6uvmqwh4      3417   \n",
       "2           2  anetv_jkn6uvmqwh4      3415   \n",
       "3           3  anetv_jkn6uvmqwh4      3417   \n",
       "4           4  anetv_Bri_myFFu4A      2408   \n",
       "\n",
       "                                         startphrase  \\\n",
       "0  Members of the procession walk down the street...   \n",
       "1  A drum line passes by walking down the street ...   \n",
       "2  A group of members in green uniforms walks wav...   \n",
       "3  A drum line passes by walking down the street ...   \n",
       "4     The person plays a song on the violin. The man   \n",
       "\n",
       "                                               sent1  \\\n",
       "0  Members of the procession walk down the street...   \n",
       "1  A drum line passes by walking down the street ...   \n",
       "2  A group of members in green uniforms walks wav...   \n",
       "3  A drum line passes by walking down the street ...   \n",
       "4             The person plays a song on the violin.   \n",
       "\n",
       "                       sent2 gold-source  \\\n",
       "0                A drum line        gold   \n",
       "1  Members of the procession         gen   \n",
       "2  Members of the procession        gold   \n",
       "3  Members of the procession         gen   \n",
       "4                    The man        gold   \n",
       "\n",
       "                                             ending0  \\\n",
       "0  passes by walking down the street playing thei...   \n",
       "1  are playing ping pong and celebrating one left...   \n",
       "2  pay the other coaches to cheer as people this ...   \n",
       "3  are playing ping pong and celebrating one left...   \n",
       "4       finishes the song and lowers the instrument.   \n",
       "\n",
       "                                             ending1  \\\n",
       "0                        has heard approaching them.   \n",
       "1                    wait slowly towards the cadets.   \n",
       "2  walk down the street holding small horn brass ...   \n",
       "3                    wait slowly towards the cadets.   \n",
       "4  hits the saxophone and demonstrates how to pro...   \n",
       "\n",
       "                                             ending2  \\\n",
       "0    arrives and they're outside dancing and asleep.   \n",
       "1  continues to play as well along the crowd alon...   \n",
       "2                         is seen in the background.   \n",
       "3  makes a square call and ends by jumping down i...   \n",
       "4  finishes massage the instrument again and cont...   \n",
       "\n",
       "                                             ending3  label  \n",
       "0     turns the lead singer watches the performance.      0  \n",
       "1           continue to play marching, interspersed.      3  \n",
       "2  are talking a couple of people playing a game ...      1  \n",
       "3  play and go back and forth hitting the drums w...      3  \n",
       "4  continues dancing while the man gore the music...      0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "swag = pd.read_csv('swag/train.csv')\n",
    "swag.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_map = {'A': '0', 'B': '1', 'C': '2', 'D': '3'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "ogdkn0uvfGez"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd9eb64eb74b446c9f40d6b23cf11c65",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "res = []\n",
    "for idx, row in tqdm(train.iterrows()):\n",
    "    content = row['Content']\n",
    "    questions = row['Questions']\n",
    "    for question in questions:\n",
    "        q_id = question['Q_id']\n",
    "        choices = question['Choices']\n",
    "        answer = question['Answer']\n",
    "        question = question['Question']\n",
    "        modified_choices = [\"\", \"\", \"\", \"\"]\n",
    "        for choice_idx, choice in enumerate(choices):\n",
    "            modified_choices[choice_idx] = choice[2:]\n",
    "        label = label_map[answer]\n",
    "        ## Hard-code for swag format!\n",
    "        res.append((\"\", \n",
    "                    \"\",\n",
    "                    q_id, \n",
    "                    \"\", \n",
    "                    content, \n",
    "                    question, \n",
    "                    \"\", \n",
    "                    modified_choices[0], \n",
    "                    modified_choices[1], \n",
    "                    modified_choices[2], \n",
    "                    modified_choices[3], \n",
    "                    label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "dymQwUJ6iKwG"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15425, 12)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>video-id</th>\n",
       "      <th>fold-ind</th>\n",
       "      <th>startphrase</th>\n",
       "      <th>sent1</th>\n",
       "      <th>sent2</th>\n",
       "      <th>gold-source</th>\n",
       "      <th>ending0</th>\n",
       "      <th>ending1</th>\n",
       "      <th>ending2</th>\n",
       "      <th>ending3</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>000101</td>\n",
       "      <td></td>\n",
       "      <td>奉和袭美抱疾杜门见寄次韵  陆龟蒙虽失春城醉上期，下帷裁遍未裁诗。因吟郢岸百亩蕙，欲采商崖三...</td>\n",
       "      <td>下列对这首诗的理解和赏析，不正确的一项是</td>\n",
       "      <td></td>\n",
       "      <td>作者写作此诗之时，皮日休正患病居家，闭门谢客，与外界不通音讯。</td>\n",
       "      <td>由于友人患病，原有的约会被暂时搁置，作者游春的诗篇也未能写出。</td>\n",
       "      <td>作者虽然身在书斋从事教学，但心中盼望能走进自然，领略美好春光。</td>\n",
       "      <td>尾联使用了关于沈约的典故，可以由此推测皮日休所患的疾病是目疾。</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>000201</td>\n",
       "      <td></td>\n",
       "      <td>隆冬之际，西伯利亚的寒流（笼罩/席卷）欧亚大陆，狂风肆虐，草木凋凌，而那些春天的元素——温暖...</td>\n",
       "      <td>1. 文中“肆虐”“凋凌”“昼夜兼程”“版图”“稚嫩”“嫣然”“抱憾退出”的词语中，有错别字...</td>\n",
       "      <td></td>\n",
       "      <td>肆虐  凋凌</td>\n",
       "      <td>集结  昼夜兼程</td>\n",
       "      <td>版图  稚嫩</td>\n",
       "      <td>嫣然  抱憾退出</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>000202</td>\n",
       "      <td></td>\n",
       "      <td>隆冬之际，西伯利亚的寒流（笼罩/席卷）欧亚大陆，狂风肆虐，草木凋凌，而那些春天的元素——温暖...</td>\n",
       "      <td>依次选用文中括号里的词语，最恰当的一项是</td>\n",
       "      <td></td>\n",
       "      <td>席卷  系  扩展</td>\n",
       "      <td>笼罩  停  扩展</td>\n",
       "      <td>席卷  停  扩充</td>\n",
       "      <td>笼罩  系  扩充</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>000301</td>\n",
       "      <td></td>\n",
       "      <td>材料一：日前，中国科学院在京召开新闻发布会对外宣布“墨子号”进行科学实验卫星提前并圆满实现全...</td>\n",
       "      <td>下列对材料相关内容的理解，不正确的一项是</td>\n",
       "      <td></td>\n",
       "      <td>量子通信把量子物理与信息技术结合起来，利用量子调控技术对信息进行编码、存储、传输和操纵，可以...</td>\n",
       "      <td>潘建伟研究团队在天宫二号空间站上进行太空量子实验，并计划发射“墨子号”后的第二颗卫星，他对未...</td>\n",
       "      <td>中国继美国、英国、日本之后成为第四个拥有散裂中子源设施的国家，有些日本科学家有了危机感，认为...</td>\n",
       "      <td>在基础科学研究领域，比如使用人造卫星开展科学实验，需要耗费巨额资金，欧洲和日本都还在犹豫不决...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>000302</td>\n",
       "      <td></td>\n",
       "      <td>材料一：日前，中国科学院在京召开新闻发布会对外宣布“墨子号”进行科学实验卫星提前并圆满实现全...</td>\n",
       "      <td>下列对材料相关内容的概括和分析，不正确的一项是</td>\n",
       "      <td></td>\n",
       "      <td>利用“墨子号”科学实验卫星研究量子密钥发布和量子隐形形态的量子通信技术，对国家信息安全和人类...</td>\n",
       "      <td>量子密钥分发是通过量子态的传输，使双方共享无条件安全的量子密钥，对信息进行一次一密的严格加密...</td>\n",
       "      <td>考虑到千百年来人们对于通信安全的追求从未停止，市场潜力巨大，中国和欧洲都投入巨额资金，首要目...</td>\n",
       "      <td>材料二和材料三中，国外媒体对我国量子通信技术研究的相关情况进行了报道，认为中国无论是投资力度...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Unnamed: 0 video-id fold-ind startphrase  \\\n",
       "0                       000101               \n",
       "1                       000201               \n",
       "2                       000202               \n",
       "3                       000301               \n",
       "4                       000302               \n",
       "\n",
       "                                               sent1  \\\n",
       "0  奉和袭美抱疾杜门见寄次韵  陆龟蒙虽失春城醉上期，下帷裁遍未裁诗。因吟郢岸百亩蕙，欲采商崖三...   \n",
       "1  隆冬之际，西伯利亚的寒流（笼罩/席卷）欧亚大陆，狂风肆虐，草木凋凌，而那些春天的元素——温暖...   \n",
       "2  隆冬之际，西伯利亚的寒流（笼罩/席卷）欧亚大陆，狂风肆虐，草木凋凌，而那些春天的元素——温暖...   \n",
       "3  材料一：日前，中国科学院在京召开新闻发布会对外宣布“墨子号”进行科学实验卫星提前并圆满实现全...   \n",
       "4  材料一：日前，中国科学院在京召开新闻发布会对外宣布“墨子号”进行科学实验卫星提前并圆满实现全...   \n",
       "\n",
       "                                               sent2 gold-source  \\\n",
       "0                               下列对这首诗的理解和赏析，不正确的一项是               \n",
       "1  1. 文中“肆虐”“凋凌”“昼夜兼程”“版图”“稚嫩”“嫣然”“抱憾退出”的词语中，有错别字...               \n",
       "2                               依次选用文中括号里的词语，最恰当的一项是               \n",
       "3                               下列对材料相关内容的理解，不正确的一项是               \n",
       "4                            下列对材料相关内容的概括和分析，不正确的一项是               \n",
       "\n",
       "                                             ending0  \\\n",
       "0                    作者写作此诗之时，皮日休正患病居家，闭门谢客，与外界不通音讯。   \n",
       "1                                             肆虐  凋凌   \n",
       "2                                          席卷  系  扩展   \n",
       "3  量子通信把量子物理与信息技术结合起来，利用量子调控技术对信息进行编码、存储、传输和操纵，可以...   \n",
       "4  利用“墨子号”科学实验卫星研究量子密钥发布和量子隐形形态的量子通信技术，对国家信息安全和人类...   \n",
       "\n",
       "                                             ending1  \\\n",
       "0                    由于友人患病，原有的约会被暂时搁置，作者游春的诗篇也未能写出。   \n",
       "1                                           集结  昼夜兼程   \n",
       "2                                          笼罩  停  扩展   \n",
       "3  潘建伟研究团队在天宫二号空间站上进行太空量子实验，并计划发射“墨子号”后的第二颗卫星，他对未...   \n",
       "4  量子密钥分发是通过量子态的传输，使双方共享无条件安全的量子密钥，对信息进行一次一密的严格加密...   \n",
       "\n",
       "                                             ending2  \\\n",
       "0                    作者虽然身在书斋从事教学，但心中盼望能走进自然，领略美好春光。   \n",
       "1                                             版图  稚嫩   \n",
       "2                                          席卷  停  扩充   \n",
       "3  中国继美国、英国、日本之后成为第四个拥有散裂中子源设施的国家，有些日本科学家有了危机感，认为...   \n",
       "4  考虑到千百年来人们对于通信安全的追求从未停止，市场潜力巨大，中国和欧洲都投入巨额资金，首要目...   \n",
       "\n",
       "                                             ending3 label  \n",
       "0                    尾联使用了关于沈约的典故，可以由此推测皮日休所患的疾病是目疾。     0  \n",
       "1                                           嫣然  抱憾退出     0  \n",
       "2                                          笼罩  系  扩充     0  \n",
       "3  在基础科学研究领域，比如使用人造卫星开展科学实验，需要耗费巨额资金，欧洲和日本都还在犹豫不决...     3  \n",
       "4  材料二和材料三中，国外媒体对我国量子通信技术研究的相关情况进行了报道，认为中国无论是投资力度...     2  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols = swag.columns.tolist()\n",
    "df = pd.DataFrame(res, columns=cols)\n",
    "\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Content</th>\n",
       "      <th>Questions</th>\n",
       "      <th>Type</th>\n",
       "      <th>Diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>周鼎传①公讳鼎，字实甫，自为童子时即以言动自持，趋立多中礼。早岁善属文，补邑诸生，坚意嗜读，...</td>\n",
       "      <td>[{'Question': '文中“自是豪强屏息”中“屏”的意思是', 'Choices':...</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>苏轼字子瞻，眉州眉山人。母程氏亲授以书，闻古今成败，辄能语其要。嘉佑二年，试礼部，主司欧阳修...</td>\n",
       "      <td>[{'Question': '下列对文中\"三年权知礼部贡举会大雪苦寒士坐庭中噤未能言轼宽其禁...</td>\n",
       "      <td>11</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>“大洋一号”是中国第一艘现在化的综合性远洋科学考察船，自1995年以来，这艘船经历了大洋款产...</td>\n",
       "      <td>[{'Question': '文中“这艘船经历了大洋款产资源研究开发专项的多个远洋调查航次和...</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>心源亭记（明）李贤     ①邓1之儒学，在古城巽隅2，规制宏敞，殿后曰“明伦堂”。堂前道中...</td>\n",
       "      <td>[{'Question': '第①段理解不正确的一项是', 'Choices': ['A、高...</td>\n",
       "      <td>11</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>谢贞，字元正，陈郡阳夏人，晋太傅安九世孙也。父蔺，正员外郎，兼散骑常侍。贞幼聪敏，有至性。祖...</td>\n",
       "      <td>[{'Question': '对下列语句中的字的解释，不正确的一项是', 'Choices'...</td>\n",
       "      <td>11</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID                                            Content  \\\n",
       "0   1  周鼎传①公讳鼎，字实甫，自为童子时即以言动自持，趋立多中礼。早岁善属文，补邑诸生，坚意嗜读，...   \n",
       "1   2  苏轼字子瞻，眉州眉山人。母程氏亲授以书，闻古今成败，辄能语其要。嘉佑二年，试礼部，主司欧阳修...   \n",
       "2   3  “大洋一号”是中国第一艘现在化的综合性远洋科学考察船，自1995年以来，这艘船经历了大洋款产...   \n",
       "3   4  心源亭记（明）李贤     ①邓1之儒学，在古城巽隅2，规制宏敞，殿后曰“明伦堂”。堂前道中...   \n",
       "4   5  谢贞，字元正，陈郡阳夏人，晋太傅安九世孙也。父蔺，正员外郎，兼散骑常侍。贞幼聪敏，有至性。祖...   \n",
       "\n",
       "                                           Questions  Type  Diff  \n",
       "0  [{'Question': '文中“自是豪强屏息”中“屏”的意思是', 'Choices':...    11     1  \n",
       "1  [{'Question': '下列对文中\"三年权知礼部贡举会大雪苦寒士坐庭中噤未能言轼宽其禁...    11     6  \n",
       "2  [{'Question': '文中“这艘船经历了大洋款产资源研究开发专项的多个远洋调查航次和...     0     4  \n",
       "3  [{'Question': '第①段理解不正确的一项是', 'Choices': ['A、高...    11     6  \n",
       "4  [{'Question': '对下列语句中的字的解释，不正确的一项是', 'Choices'...    11     6  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = pd.read_json('raw_data/validation.json')\n",
    "\n",
    "print(test.shape)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98e1c4f0e76846c283a3e9c4003da952",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "res = []\n",
    "for idx, row in tqdm(test.iterrows()):\n",
    "    content = row['Content']\n",
    "    questions = row['Questions']\n",
    "    for question in questions:\n",
    "        q_id = question['Q_id']\n",
    "        choices = question['Choices']\n",
    "        question = question['Question']\n",
    "        modified_choices = [\"\", \"\", \"\", \"\"]\n",
    "        for choice_idx, choice in enumerate(choices):\n",
    "            modified_choices[choice_idx] = choice[2:]\n",
    "        ## Hard-code for swag format!\n",
    "        res.append((\"\", \n",
    "                    \"\",\n",
    "                    q_id, \n",
    "                    \"\", \n",
    "                    content, \n",
    "                    question, \n",
    "                    \"\", \n",
    "                    modified_choices[0], \n",
    "                    modified_choices[1], \n",
    "                    modified_choices[2], \n",
    "                    modified_choices[3],\n",
    "                    0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2444, 12)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>video-id</th>\n",
       "      <th>fold-ind</th>\n",
       "      <th>startphrase</th>\n",
       "      <th>sent1</th>\n",
       "      <th>sent2</th>\n",
       "      <th>gold-source</th>\n",
       "      <th>ending0</th>\n",
       "      <th>ending1</th>\n",
       "      <th>ending2</th>\n",
       "      <th>ending3</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>000101</td>\n",
       "      <td></td>\n",
       "      <td>周鼎传①公讳鼎，字实甫，自为童子时即以言动自持，趋立多中礼。早岁善属文，补邑诸生，坚意嗜读，...</td>\n",
       "      <td>文中“自是豪强屏息”中“屏”的意思是</td>\n",
       "      <td></td>\n",
       "      <td>屏住呼吸</td>\n",
       "      <td>收敛行迹</td>\n",
       "      <td>退避迁徙</td>\n",
       "      <td>抑止打压</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>000102</td>\n",
       "      <td></td>\n",
       "      <td>周鼎传①公讳鼎，字实甫，自为童子时即以言动自持，趋立多中礼。早岁善属文，补邑诸生，坚意嗜读，...</td>\n",
       "      <td>文中“多意沮而寝”中“寝”的意思是</td>\n",
       "      <td></td>\n",
       "      <td>睡觉</td>\n",
       "      <td>丑陋</td>\n",
       "      <td>隐蔽</td>\n",
       "      <td>停止</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>000103</td>\n",
       "      <td></td>\n",
       "      <td>周鼎传①公讳鼎，字实甫，自为童子时即以言动自持，趋立多中礼。早岁善属文，补邑诸生，坚意嗜读，...</td>\n",
       "      <td>第④段断句正确的一项是</td>\n",
       "      <td></td>\n",
       "      <td>立捕两奴/及舟械至奴/犹称相国/从者意不下。</td>\n",
       "      <td>立捕两奴/及舟/械至奴/犹称相国/从者意不下。</td>\n",
       "      <td>立捕两奴/及舟械至奴/犹称相国从者/意不下。</td>\n",
       "      <td>立捕两奴及舟械/至奴/犹称相国从者/意不下。</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>000201</td>\n",
       "      <td></td>\n",
       "      <td>苏轼字子瞻，眉州眉山人。母程氏亲授以书，闻古今成败，辄能语其要。嘉佑二年，试礼部，主司欧阳修...</td>\n",
       "      <td>下列对文中\"三年权知礼部贡举会大雪苦寒士坐庭中噤未能言轼宽其禁约使得尽技巡铺内侍每摧辱举子且...</td>\n",
       "      <td></td>\n",
       "      <td>三年/权知礼部贡举/会大雪苦寒/士坐庭中/噤未能言/轼宽其禁约/使得尽技/巡辅内侍每摧辱举子...</td>\n",
       "      <td>三年/权知礼部贡举/会大雪苦寒/士坐庭中噤/未能言轼/宽其禁约/使得尽技/巡辅内侍每摧辱举...</td>\n",
       "      <td>三年/权知礼部贡举/会大雪苦寒/士坐庭中噤/未能言轼/宽其禁约/使得尽技巡辅内侍/每摧辱举...</td>\n",
       "      <td>三年/权知礼部贡举/会大雪苦寒/士坐庭中/噤未能言/轼宽其禁约/使得尽技巡辅内侍/每摧辱举...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>000202</td>\n",
       "      <td></td>\n",
       "      <td>苏轼字子瞻，眉州眉山人。母程氏亲授以书，闻古今成败，辄能语其要。嘉佑二年，试礼部，主司欧阳修...</td>\n",
       "      <td>下列对原文有关内容的概括和分析，不正确的一项是</td>\n",
       "      <td></td>\n",
       "      <td>苏轼自幼聪颖，深受时贤赏识。母亲亲自为他授课，他往往能说出要点。欧阳修十分看重他，曾对梅圣俞...</td>\n",
       "      <td>苏轼因势利导，利用新法便民。当时王安石创行新法，他上书论其不便；新政下达，他常常设法使这些法...</td>\n",
       "      <td>苏轼直面饥疫，解救受灾百姓。他在任职杭州时遭遇旱灾病疫，减免上供米三分之一纾缓灾情；同时又集...</td>\n",
       "      <td>苏轼天赋异禀，为文得心应手。他从父习文，又极具才华，作文如行云流水，行止有度，嬉笑怒骂之辞，...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Unnamed: 0 video-id fold-ind startphrase  \\\n",
       "0                       000101               \n",
       "1                       000102               \n",
       "2                       000103               \n",
       "3                       000201               \n",
       "4                       000202               \n",
       "\n",
       "                                               sent1  \\\n",
       "0  周鼎传①公讳鼎，字实甫，自为童子时即以言动自持，趋立多中礼。早岁善属文，补邑诸生，坚意嗜读，...   \n",
       "1  周鼎传①公讳鼎，字实甫，自为童子时即以言动自持，趋立多中礼。早岁善属文，补邑诸生，坚意嗜读，...   \n",
       "2  周鼎传①公讳鼎，字实甫，自为童子时即以言动自持，趋立多中礼。早岁善属文，补邑诸生，坚意嗜读，...   \n",
       "3  苏轼字子瞻，眉州眉山人。母程氏亲授以书，闻古今成败，辄能语其要。嘉佑二年，试礼部，主司欧阳修...   \n",
       "4  苏轼字子瞻，眉州眉山人。母程氏亲授以书，闻古今成败，辄能语其要。嘉佑二年，试礼部，主司欧阳修...   \n",
       "\n",
       "                                               sent2 gold-source  \\\n",
       "0                                 文中“自是豪强屏息”中“屏”的意思是               \n",
       "1                                  文中“多意沮而寝”中“寝”的意思是               \n",
       "2                                        第④段断句正确的一项是               \n",
       "3  下列对文中\"三年权知礼部贡举会大雪苦寒士坐庭中噤未能言轼宽其禁约使得尽技巡铺内侍每摧辱举子且...               \n",
       "4                            下列对原文有关内容的概括和分析，不正确的一项是               \n",
       "\n",
       "                                             ending0  \\\n",
       "0                                               屏住呼吸   \n",
       "1                                                 睡觉   \n",
       "2                             立捕两奴/及舟械至奴/犹称相国/从者意不下。   \n",
       "3  三年/权知礼部贡举/会大雪苦寒/士坐庭中/噤未能言/轼宽其禁约/使得尽技/巡辅内侍每摧辱举子...   \n",
       "4  苏轼自幼聪颖，深受时贤赏识。母亲亲自为他授课，他往往能说出要点。欧阳修十分看重他，曾对梅圣俞...   \n",
       "\n",
       "                                             ending1  \\\n",
       "0                                               收敛行迹   \n",
       "1                                                 丑陋   \n",
       "2                            立捕两奴/及舟/械至奴/犹称相国/从者意不下。   \n",
       "3   三年/权知礼部贡举/会大雪苦寒/士坐庭中噤/未能言轼/宽其禁约/使得尽技/巡辅内侍每摧辱举...   \n",
       "4  苏轼因势利导，利用新法便民。当时王安石创行新法，他上书论其不便；新政下达，他常常设法使这些法...   \n",
       "\n",
       "                                             ending2  \\\n",
       "0                                               退避迁徙   \n",
       "1                                                 隐蔽   \n",
       "2                             立捕两奴/及舟械至奴/犹称相国从者/意不下。   \n",
       "3   三年/权知礼部贡举/会大雪苦寒/士坐庭中噤/未能言轼/宽其禁约/使得尽技巡辅内侍/每摧辱举...   \n",
       "4  苏轼直面饥疫，解救受灾百姓。他在任职杭州时遭遇旱灾病疫，减免上供米三分之一纾缓灾情；同时又集...   \n",
       "\n",
       "                                             ending3  label  \n",
       "0                                               抑止打压      0  \n",
       "1                                                 停止      0  \n",
       "2                             立捕两奴及舟械/至奴/犹称相国从者/意不下。      0  \n",
       "3   三年/权知礼部贡举/会大雪苦寒/士坐庭中/噤未能言/轼宽其禁约/使得尽技巡辅内侍/每摧辱举...      0  \n",
       "4  苏轼天赋异禀，为文得心应手。他从父习文，又极具才华，作文如行云流水，行止有度，嬉笑怒骂之辞，...      0  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test = pd.DataFrame(res, columns=cols)\n",
    "\n",
    "print(df_test.shape)\n",
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "hkGRznM0hiS0"
   },
   "outputs": [],
   "source": [
    "if DEBUG:\n",
    "    df.iloc[:50].to_csv('train.csv', index=False)\n",
    "    df.iloc[-50:].to_csv('valid.csv', index=False)\n",
    "    df_test.iloc[:50].to_csv('test.csv', index=False)\n",
    "else:\n",
    "    df.iloc[:13000].to_csv('train.csv', index=False)\n",
    "    df.iloc[13000:].to_csv('valid.csv', index=False)\n",
    "    df_test.to_csv('test.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !rm -rf /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fYpIkMo9hrjh",
    "outputId": "d37b581b-6ff2-4124-c3d0-148a43e7f34a",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "02/18/2021 09:38:56 - WARNING - __main__ -   Process rank: -1, device: cuda:0, n_gpu: 1distributed training: False, 16-bits training: False\n",
      "02/18/2021 09:38:56 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir=/media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta, overwrite_output_dir=True, do_train=True, do_eval=True, do_predict=True, evaluation_strategy=EvaluationStrategy.NO, prediction_loss_only=False, per_device_train_batch_size=2, per_device_eval_batch_size=2, gradient_accumulation_steps=8, eval_accumulation_steps=None, learning_rate=2e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=5.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_steps=0, logging_dir=runs/Feb18_09-38-56_user01-X9DRG-HF, logging_first_step=False, logging_steps=500, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=False, fp16_opt_level=O1, fp16_backend=auto, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name=/media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=False, deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, report_to=['tensorboard', 'wandb'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, _n_gpu=1)\n",
      "Using custom data configuration default\n",
      "Downloading and preparing dataset csv/default-d0744900419e6880 (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /home/user01/.cache/huggingface/datasets/csv/default-d0744900419e6880/0.0.0/2960f95a26e85d40ca41a230ac88787f715ee3003edaacb8b1f0891e9f04dda2...\n",
      "Dataset csv downloaded and prepared to /home/user01/.cache/huggingface/datasets/csv/default-d0744900419e6880/0.0.0/2960f95a26e85d40ca41a230ac88787f715ee3003edaacb8b1f0891e9f04dda2. Subsequent calls will reuse this data.\n",
      "loading configuration file https://huggingface.co/hfl/chinese-roberta-wwm-ext/resolve/main/config.json from cache at /home/user01/.cache/huggingface/transformers/b64aa51c20341fe5461d0663677dd1527cc8fb71c482d06ee75406857d0ed53a.1d3cb63600c62f190470f1f1b36eaa1689928fa06d586702865792c3a2575c0f\n",
      "Model config BertConfig {\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"eos_token_id\": 2,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.4.0.dev0\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 21128\n",
      "}\n",
      "\n",
      "loading configuration file https://huggingface.co/hfl/chinese-roberta-wwm-ext/resolve/main/config.json from cache at /home/user01/.cache/huggingface/transformers/b64aa51c20341fe5461d0663677dd1527cc8fb71c482d06ee75406857d0ed53a.1d3cb63600c62f190470f1f1b36eaa1689928fa06d586702865792c3a2575c0f\n",
      "Model config BertConfig {\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"eos_token_id\": 2,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.4.0.dev0\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 21128\n",
      "}\n",
      "\n",
      "Model name 'hfl/chinese-roberta-wwm-ext' not found in model shortcut name list (bert-base-uncased, bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, bert-base-multilingual-cased, bert-base-chinese, bert-base-german-cased, bert-large-uncased-whole-word-masking, bert-large-cased-whole-word-masking, bert-large-uncased-whole-word-masking-finetuned-squad, bert-large-cased-whole-word-masking-finetuned-squad, bert-base-cased-finetuned-mrpc, bert-base-german-dbmdz-cased, bert-base-german-dbmdz-uncased, TurkuNLP/bert-base-finnish-cased-v1, TurkuNLP/bert-base-finnish-uncased-v1, wietsedv/bert-base-dutch-cased). Assuming 'hfl/chinese-roberta-wwm-ext' is a path, a model identifier, or url to a directory containing tokenizer files.\n",
      "loading file https://huggingface.co/hfl/chinese-roberta-wwm-ext/resolve/main/vocab.txt from cache at /home/user01/.cache/huggingface/transformers/92a56e79ec6564fd501527ed88ca336637eb4bfeb28d10580c3bbdfb7889a032.accd894ff58c6ff7bd4f3072890776c14f4ea34fcc08e79cd88c2d157756dceb\n",
      "loading file https://huggingface.co/hfl/chinese-roberta-wwm-ext/resolve/main/tokenizer.json from cache at None\n",
      "loading file https://huggingface.co/hfl/chinese-roberta-wwm-ext/resolve/main/added_tokens.json from cache at /home/user01/.cache/huggingface/transformers/87c7eedd995b4bae2c34df3baf2cbd5df5496bed675126427849c72e590f5574.5cc6e825eb228a7a5cfd27cb4d7151e97a79fb962b31aaf1813aa102e746584b\n",
      "loading file https://huggingface.co/hfl/chinese-roberta-wwm-ext/resolve/main/special_tokens_map.json from cache at /home/user01/.cache/huggingface/transformers/d521373fc7ac35f63d56cf303de74a202403dcf1aaa792cd01f653694be59563.dd8bd9bfd3664b530ea4e645105f557769387b3da9f79bdb55ed556bdd80611d\n",
      "loading file https://huggingface.co/hfl/chinese-roberta-wwm-ext/resolve/main/tokenizer_config.json from cache at /home/user01/.cache/huggingface/transformers/5dedf24c46ec573f8a27ddfa6e737869ec8df462a9a7682b35ded34301c5bdc8.d23f50bbddc3fb34db5a76d47fa9bdd5d75bf4201ad2d49abbcca25629b3e562\n",
      "loading weights file https://huggingface.co/hfl/chinese-roberta-wwm-ext/resolve/main/pytorch_model.bin from cache at /home/user01/.cache/huggingface/transformers/ebc33cec9cd4890c20bd3b688fbf8e907167e0e2f209b801b3159123cd4630e4.d863eb12d1b0d00e5d41e9eb0d41914e4993c03e6de69e67bc10c79818f5fd4d\n",
      "Some weights of the model checkpoint at hfl/chinese-roberta-wwm-ext were not used when initializing BertForMultipleChoice: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMultipleChoice from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMultipleChoice from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForMultipleChoice were not initialized from the model checkpoint at hfl/chinese-roberta-wwm-ext and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "100%|███████████████████████████████████████████| 13/13 [00:30<00:00,  2.32s/ba]\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:06<00:00,  2.04s/ba]\n",
      "100%|█████████████████████████████████████████████| 3/3 [00:05<00:00,  1.89s/ba]\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "W&B installed but not logged in. Run `wandb login` or set the WANDB_API_KEY env variable.\n",
      "The following columns in the training set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: Unnamed: 0, ending3, fold-ind, video-id, gold-source, sent2, ending1, ending2, ending0, startphrase, sent1.\n",
      "The following columns in the evaluation set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: Unnamed: 0, ending3, fold-ind, video-id, gold-source, sent2, ending1, ending2, ending0, startphrase, sent1.\n",
      "***** Running training *****\n",
      "  Num examples = 13000\n",
      "  Num Epochs = 5\n",
      "  Instantaneous batch size per device = 2\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
      "  Gradient Accumulation steps = 8\n",
      "  Total optimization steps = 4060\n",
      "{'loss': 1.3942, 'learning_rate': 1.7536945812807884e-05, 'epoch': 0.62}        \n",
      " 12%|████▋                                 | 500/4060 [19:10<2:16:34,  2.30s/it]Saving model checkpoint to /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-500\n",
      "Configuration saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-500/config.json\n",
      "Model weights saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-500/pytorch_model.bin\n",
      "{'loss': 1.3936, 'learning_rate': 1.5073891625615764e-05, 'epoch': 1.23}        \n",
      " 25%|█████████                            | 1000/4060 [38:27<1:57:43,  2.31s/it]Saving model checkpoint to /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-1000\n",
      "Configuration saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-1000/config.json\n",
      "Model weights saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-1000/pytorch_model.bin\n",
      "{'loss': 1.367, 'learning_rate': 1.2610837438423646e-05, 'epoch': 1.85}         \n",
      " 37%|█████████████▋                       | 1500/4060 [57:41<1:37:40,  2.29s/it]Saving model checkpoint to /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-1500\n",
      "Configuration saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-1500/config.json\n",
      "Model weights saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-1500/pytorch_model.bin\n",
      "{'loss': 1.2811, 'learning_rate': 1.0147783251231529e-05, 'epoch': 2.46}        \n",
      " 49%|█████████████████▏                 | 2000/4060 [1:16:55<1:19:13,  2.31s/it]Saving model checkpoint to /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-2000\n",
      "Configuration saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-2000/config.json\n",
      "Model weights saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-2000/pytorch_model.bin\n",
      "{'loss': 1.2039, 'learning_rate': 7.68472906403941e-06, 'epoch': 3.08}          \n",
      " 62%|█████████████████████▌             | 2500/4060 [1:36:09<1:00:20,  2.32s/it]Saving model checkpoint to /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-2500\n",
      "Configuration saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-2500/config.json\n",
      "Model weights saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-2500/pytorch_model.bin\n",
      "{'loss': 1.067, 'learning_rate': 5.2216748768472915e-06, 'epoch': 3.69}         \n",
      " 74%|███████████████████████████▎         | 3000/4060 [1:55:17<40:43,  2.31s/it]Saving model checkpoint to /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-3000\n",
      "Configuration saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-3000/config.json\n",
      "Model weights saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-3000/pytorch_model.bin\n",
      "{'loss': 0.9923, 'learning_rate': 2.7586206896551725e-06, 'epoch': 4.31}        \n",
      " 86%|███████████████████████████████▉     | 3500/4060 [2:14:32<21:37,  2.32s/it]Saving model checkpoint to /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-3500\n",
      "Configuration saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-3500/config.json\n",
      "Model weights saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-3500/pytorch_model.bin\n",
      "{'loss': 0.9081, 'learning_rate': 2.955665024630542e-07, 'epoch': 4.93}         \n",
      " 99%|████████████████████████████████████▍| 4000/4060 [2:33:51<02:19,  2.32s/it]Saving model checkpoint to /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-4000\n",
      "Configuration saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-4000/config.json\n",
      "Model weights saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/checkpoint-4000/pytorch_model.bin\n",
      "100%|█████████████████████████████████████| 4060/4060 [2:36:11<00:00,  2.31s/it]\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "{'train_runtime': 9371.807, 'train_samples_per_second': 0.433, 'epoch': 5.0}    \n",
      "100%|█████████████████████████████████████| 4060/4060 [2:36:11<00:00,  2.31s/it]\n",
      "Saving model checkpoint to /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta\n",
      "Configuration saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/config.json\n",
      "Model weights saved in /media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/pytorch_model.bin\n",
      "02/18/2021 12:16:05 - INFO - __main__ -   ***** Train results *****\n",
      "02/18/2021 12:16:05 - INFO - __main__ -     epoch = 5.0\n",
      "02/18/2021 12:16:05 - INFO - __main__ -     train_runtime = 9371.807\n",
      "02/18/2021 12:16:05 - INFO - __main__ -     train_samples_per_second = 0.433\n",
      "02/18/2021 12:16:05 - INFO - __main__ -   *** Evaluate ***\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2425\n",
      "  Batch size = 2\n",
      "100%|███████████████████████████████████████| 1213/1213 [01:50<00:00, 11.01it/s]\n",
      "02/18/2021 12:17:55 - INFO - __main__ -   ***** Eval results *****\n",
      "02/18/2021 12:17:55 - INFO - __main__ -     epoch = 5.0\n",
      "02/18/2021 12:17:55 - INFO - __main__ -     eval_accuracy = 0.4342268109321594\n",
      "02/18/2021 12:17:55 - INFO - __main__ -     eval_loss = 1.4691760540008545\n",
      "02/18/2021 12:17:55 - INFO - __main__ -     eval_runtime = 110.2303\n",
      "02/18/2021 12:17:55 - INFO - __main__ -     eval_samples_per_second = 21.999\n",
      "02/18/2021 12:17:55 - INFO - __main__ -   *** Test ***\n",
      "The following columns in the test set don't have a corresponding argument in `BertForMultipleChoice.forward` and have been ignored: Unnamed: 0, ending3, fold-ind, video-id, gold-source, sent2, ending1, ending2, ending0, startphrase, sent1.\n",
      "***** Running Prediction *****\n",
      "  Num examples = 2444\n",
      "  Batch size = 2\n",
      "100%|███████████████████████████████████████| 1222/1222 [01:53<00:00, 11.57it/s]02/18/2021 12:19:49 - INFO - __main__ -   ***** Test results *****\n",
      "100%|███████████████████████████████████████| 1222/1222 [01:53<00:00, 10.75it/s]\n"
     ]
    }
   ],
   "source": [
    "!python ./transformers/examples/multiple-choice/run_swag.py \\\n",
    "--model_name_or_path 'hfl/chinese-roberta-wwm-ext' \\\n",
    "--do_train \\\n",
    "--do_eval \\\n",
    "--do_predict \\\n",
    "--max_seq_length 512 \\\n",
    "--train_file train.csv \\\n",
    "--validation_file valid.csv \\\n",
    "--test_file test.csv \\\n",
    "--learning_rate 2e-5 \\\n",
    "--num_train_epochs 5 \\\n",
    "--output_dir '/media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta' \\\n",
    "--gradient_accumulation_steps 8 \\\n",
    "--per_device_eval_batch_size 2 \\\n",
    "--per_device_train_batch_size 2 \\\n",
    "--overwrite_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2444, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id label\n",
       "0   0     C\n",
       "1   1     B\n",
       "2   2     C\n",
       "3   3     A\n",
       "4   4     D"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_results = pd.read_csv('/media/user01/67a19c61-e7f0-4409-ac85-5eee69511615/haihua_roberta/test_results.txt')\n",
    "\n",
    "print(test_results.shape)\n",
    "test_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2444, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>101</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>102</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>103</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>201</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>202</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id label\n",
       "0  101     C\n",
       "1  102     B\n",
       "2  103     C\n",
       "3  201     A\n",
       "4  202     D"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub = pd.read_csv('raw_data/sample.csv')\n",
    "sub['label'] = test_results['label']\n",
    "\n",
    "print(sub.shape)\n",
    "sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub.to_csv('baseline_roberta.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "Biendata EDA",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
